{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from keras import backend as K\n",
    "from keras import applications\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras import optimizers\n",
    "from keras.models import Sequential, Model, load_model \n",
    "from keras.layers import Dropout, Flatten, Dense, GlobalAveragePooling2D, Input\n",
    "from keras import backend as k \n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping\n",
    "from keras.optimizers import Adam, RMSprop, SGD\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model_weights(model,path):\n",
    "    model.save_weights(path)\n",
    "    return\n",
    "def save_model_arch(model,path):\n",
    "    model.save(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "initial_model = applications.VGG19(weights=\"imagenet\", include_top=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv4 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv4 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv4 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "predictions (Dense)          (None, 1000)              4097000   \n",
      "=================================================================\n",
      "Total params: 143,667,240\n",
      "Trainable params: 143,667,240\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "initial_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = Dense(2, activation='softmax',name='classifier')(initial_model.layers[-2].output)\n",
    "tune_model = Model(initial_model.input, x)\n",
    "opt = Adam(lr=0.001)\n",
    "tune_model.compile(optimizer=opt,\n",
    "              loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "#sparse_categorical_crossentropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv4 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv4 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv4 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "_________________________________________________________________\n",
      "classifier (Dense)           (None, 2)                 8194      \n",
      "=================================================================\n",
      "Total params: 139,578,434\n",
      "Trainable params: 139,578,434\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tune_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1242 images belonging to 2 classes.\n",
      "Found 52 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "path = 'data/'\n",
    "target_size = (224, 224)\n",
    "batch_size = 10\n",
    "def get_batches(directory, target_size=target_size, batch_size=batch_size, shuffle=False):\n",
    " datagen = ImageDataGenerator()\n",
    " return datagen.flow_from_directory(directory=directory,\n",
    "                                    target_size=target_size,\n",
    "                                    batch_size=batch_size,\n",
    "                                    class_mode='categorical',\n",
    "                                    shuffle=shuffle)\n",
    "batches = get_batches(path+'train', shuffle=True)\n",
    "valid_batches = get_batches(path+'validation', shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "125/124 [==============================] - 1822s - loss: 7.9500 - acc: 0.5032 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 2/30\n",
      "125/124 [==============================] - 1822s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 3/30\n",
      "125/124 [==============================] - 1823s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 4/30\n",
      "125/124 [==============================] - 1831s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 5/30\n",
      "125/124 [==============================] - 1823s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 6/30\n",
      "125/124 [==============================] - 1825s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 7/30\n",
      "125/124 [==============================] - 1833s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 8/30\n",
      "125/124 [==============================] - 1829s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 9/30\n",
      "125/124 [==============================] - 1822s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 10/30\n",
      "125/124 [==============================] - 1823s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 11/30\n",
      "125/124 [==============================] - 1826s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 12/30\n",
      "125/124 [==============================] - 1823s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 13/30\n",
      "125/124 [==============================] - 1831s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 14/30\n",
      "125/124 [==============================] - 1827s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 15/30\n",
      "125/124 [==============================] - 1828s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 16/30\n",
      "125/124 [==============================] - 1830s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 17/30\n",
      "125/124 [==============================] - 1828s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 18/30\n",
      "125/124 [==============================] - 1822s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 19/30\n",
      "125/124 [==============================] - 1826s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 20/30\n",
      "125/124 [==============================] - 1827s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 21/30\n",
      "125/124 [==============================] - 1832s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 22/30\n",
      "125/124 [==============================] - 1832s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 23/30\n",
      "125/124 [==============================] - 1824s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 24/30\n",
      "125/124 [==============================] - 1824s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 25/30\n",
      "125/124 [==============================] - 1830s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 26/30\n",
      "125/124 [==============================] - 1830s - loss: 8.0462 - acc: 0.5008 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 27/30\n",
      "125/124 [==============================] - 1820s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 28/30\n",
      "125/124 [==============================] - 1824s - loss: 7.9950 - acc: 0.5040 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 29/30\n",
      "125/124 [==============================] - 1827s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n",
      "Epoch 30/30\n",
      "125/124 [==============================] - 1824s - loss: 8.0973 - acc: 0.4976 - val_loss: 7.7491 - val_acc: 0.5192\n"
     ]
    }
   ],
   "source": [
    "'''out=tune_model.fit_generator(batches, samples_per_epoch=batches.nb_sample,nb_epoch=3, \n",
    "                         validation_data=valid_batches,\n",
    "                         nb_val_samples=valid_batches.nb_sample)'''\n",
    "out=tune_model.fit_generator(batches, steps_per_epoch=batches.samples/batches.batch_size, epochs=30, validation_data=valid_batches, validation_steps=valid_batches.samples/valid_batches.batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "tune_model.save('tune_model_10eph.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
